# -*- coding: utf-8 -*- 
import urllib2
import cookielib
import time


class NetReturn(object):#下载后返回的页面,一个页面一个对象
    def __init__(self):
        self.from_task=None
        self.to_spider=None
        self.contents=None#返回的页面
        self.cookie=None#返回需要下一次任务使用的cookie

class Downloader(object):#下载类基类
    def __init__(self):
        self.netreturns=[]#用来记录返回的下载件组
        self.to_spider=None
        
class Downloader_Common(Downloader):
    def __init__(self,task):
        Downloader.__init__(self)
        self.__go(task)
        if task.to_spider!=None:
            self.to_spider=task.to_spider
        
    def __go(self,task):
        #print 'ok'
        netreturn=NetReturn()
        netreturn.task=task
        if netreturn.task.cookie!=None:
            cj=netreturn.task.cookie
        else:
            cj = cookielib.CookieJar()
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        req=urllib2.Request(task.url)
        if task.header!=None:
            for t,v in task.header.items():
                req.add_header(t,v)
        try:
            contents=opener.open(req).read()
        except:
            time.sleep(3)
            contents=opener.open(req).read()
        netreturn.contents=contents.decode('gbk').encode('utf-8')
        self.netreturns.append(netreturn)
        #time.sleep(3)
        print netreturn.contents
        


        
